
# Libraries
import pandas as pd
from matplotlib import pyplot as plt 
from sklearn.model_selection import train_test_split
import re
import string
from sklearn import preprocessing
import nltk
from nltk.tokenize import word_tokenize
from collections import Counter


#nltk.download('punkt')
#nltk.download('stopwords')


# Data reading
raw_data_df = pd.read_csv("../data/raw/fake_or_real_news.csv")


raw_data_df.head()


def classes_overview(df = None, obj_val = ""):
    '''
    Devuelve un dataframe con el porcentaje y valor abosoluto de las clases de la variable objetivo.
    :param df: Datase
    :param obj_val: Variable objetivo
    :return: Dataframe
    '''
    temp = df[obj_val].value_counts(normalize=True).mul(100).rename('percentaje').reset_index()
    temp_conteo = df[obj_val].value_counts().reset_index()
    return pd.merge(temp, temp_conteo, on=['index'], how='inner')


classes_overview(raw_data_df, 'label')


# We divide values of the target label and the rest of data
X = raw_data_df.drop('label', axis=1)
Y = raw_data_df['label']

# We make the split between train and test
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.15, random_state=1234, stratify=Y)

# We save the original data divided
train_data_df = X_train
train_data_df['label'] = Y_train
train_data_df.to_csv("../data/interim/train.csv")

test_data_df = X_test
test_data_df['label'] = Y_test
test_data_df.to_csv("../data/interim/test.csv")


print('Total de muestras del dataset: ', len(train_data_df.index))

Total de muestras del dataset:  5384


# We check the balance of the dataset again to be sure the balance is correct
classes_overview(train_data_df, 'label')


# We check the existance of anomalies in the data like NULL values
temp_df = train_data_df.isnull().sum().reset_index()
temp_df.columns = ['column_name', 'NAs']
temp_df


# We check any duplicates
sum(train_data_df.duplicated())

0


lb = preprocessing.LabelEncoder()
train_data_df['label'] = lb.fit_transform(train_data_df['label'])
print(lb.classes_)

# Repeat encoding for test sample
test_data_df['label'] = lb.transform(test_data_df['label'])

['FAKE' 'REAL']


train_data_df


translator = str.maketrans('', '', string.punctuation)
train_data_df['proc_title'] = train_data_df.title.map(lambda x : re.sub(r"[^a-zA-Z0-9]", " ",x.strip().lower().translate(translator)))
train_data_df['proc_text'] = train_data_df.text.map(lambda x : re.sub(r"[^a-zA-Z0-9]", " ",x.strip().lower().translate(translator)))

test_data_df['proc_title'] = test_data_df.title.map(lambda x : re.sub(r"[^a-zA-Z0-9]", " ",x.strip().lower().translate(translator)))
test_data_df['proc_text'] = test_data_df.text.map(lambda x : re.sub(r"[^a-zA-Z0-9]", " ",x.strip().lower().translate(translator)))


train_data_df['proc_title']

1342    voter dreading being sent over to visibly stup...
1251    abortion bill dropped amid concerns of female ...
804     internet is on fire with speculation that pode...
47      senate race rankings dems attack as gop lays s...
3888    starbucks baristas stop writing race together ...
                              ...                        
502                         romneys timely trump trolling
2824    megyn kelly interrogates tom cotton on iran le...
1768    controversial new  antifamine  gmo potato stra...
4116    meter reader knocks on man s door reveals sini...
4944     on contact  chris hedges and medea benjamin o...
Name: proc_title, Length: 5384, dtype: object


corpus_fake = ' '.join(train_data_df[train_data_df['label'] == 0].proc_title)
all_words_fake = word_tokenize(corpus_fake)
corpus_true = ' '.join(train_data_df[train_data_df['label'] == 1].proc_title)
all_words_true = word_tokenize(corpus_true)

# Cleaning all stopwords
stopwords = nltk.corpus.stopwords.words('english')
all_words_fake = [word for word in all_words_fake if word not in stopwords]
all_words_true = [word for word in all_words_true if word not in stopwords]


fig, axes = plt.subplots(1, 2, figsize=(10, 10), sharey=False)
ax1 = pd.Series(Counter(all_words_fake)).sort_values(ascending=False).iloc[:30].plot(kind="bar", ax=axes[0]);
ax1.title.set_text("Fake News Top 30 words by title")
ax2 = pd.Series(Counter(all_words_true)).sort_values(ascending=False).iloc[:30].plot(kind="bar", ax=axes[1]);
ax2.title.set_text("True News Top 30 words by title")


titles_len = [len(i.split()) for i in train_data_df['proc_title']]

pd.Series(titles_len).hist(bins = 60);
plt.xlabel('Number of Words');
plt.ylabel('Number of Texts');


corpus_fake = ' '.join(train_data_df[train_data_df['label'] == 0].proc_text)
all_words_fake = word_tokenize(corpus_fake)
corpus_true = ' '.join(train_data_df[train_data_df['label'] == 1].proc_text)
all_words_true = word_tokenize(corpus_true)

# Cleaning all stopwords
stopwords = nltk.corpus.stopwords.words('english')
all_words_fake = [word for word in all_words_fake if word not in stopwords]
all_words_true = [word for word in all_words_true if word not in stopwords]

fig, axes = plt.subplots(1, 2, figsize=(10, 10), sharey=False)
ax1 = pd.Series(Counter(all_words_fake)).sort_values(ascending=False).iloc[:30].plot(kind="bar", ax=axes[0]);
ax1.title.set_text("Fake News Top 30 words by content")
ax2 = pd.Series(Counter(all_words_true)).sort_values(ascending=False).iloc[:30].plot(kind="bar", ax=axes[1]);
ax2.title.set_text("True News Top 30 words by content")


# Libraries
import pandas as pd
import string
import re
import nltk
from nltk.stem import WordNetLemmatizer
from sklearn import preprocessing
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer


#nltk.download('wordnet')


# Train data
X_train = pd.read_csv("../data/interim/train.csv")
Y_train = X_train['label']
X_train = X_train['title']

# Test data
X_test = pd.read_csv("../data/interim/test.csv")
Y_test = X_test['label']
X_test = X_test['title']


# Translator to clean data 
translator = str.maketrans('', '', string.punctuation)

# Word lemmatizer
wordnet_lemmatizer = WordNetLemmatizer()

# Stopwords
stopwords = nltk.corpus.stopwords.words('english')


def title_cleaner(tlt):
    """
    Cleans the text passed. All text to lower, blanck spaces and punctuation.
    Also eliminates stopwords and lematize verbs
    :param tlt: text to clean
    :type tlt: string
    """
    # Lower letters and blank spaces out, we eliminate all type of punctuation
    # Finally we split the data (tokenize)
    tokens = re.sub(r"[^a-zA-Z0-9]", " ",tlt.strip().lower().translate(translator)).split()

    # Lemmatize and eliminate any kind of stopword that doesn't gives us any information
    final_words = [wordnet_lemmatizer.lemmatize(word, pos="v") for word in tokens if not word in stopwords]

    # Joining the resulted words cleaned
    return " ".join(final_words)


X_train = X_train.map(lambda x : title_cleaner(x))
X_test = X_test.map(lambda x : title_cleaner(x))


lb = preprocessing.LabelEncoder()
Y_train = lb.fit_transform(Y_train)
Y_test = lb.transform(Y_test)

print(lb.classes_)

['FAKE' 'REAL']


# Model for creating variables
corpus = X_train
cv = CountVectorizer()
tft = TfidfTransformer()

# Creating the train result 
cv_matrix = cv.fit_transform(corpus)
tfd_result = tft.fit_transform(cv_matrix)


# Preparing train data
# BoW only
features_cv= cv.get_feature_names_out()
X_train_bow = pd.DataFrame(cv_matrix.toarray(), columns=features_cv)

# Tfd added
features_tft = tft.get_feature_names_out()
X_train_tfd = pd.DataFrame(tfd_result.toarray(), columns=features_tft)


# Preparing test data
# BoW only
data_matrix = cv.transform(X_test) 
X_test_bow = pd.DataFrame(data_matrix.toarray(), columns=features_cv)

# Tfd added
X_test_tfd = pd.DataFrame(tft.transform(data_matrix).toarray(), columns=features_tft)


filenames = ["train_bow.csv", "train_tfd.csv", "test_bow.csv", "test_tfd.csv"]
samples = [X_train_bow, X_train_tfd, X_test_bow, X_test_tfd]
targets = [Y_train, Y_train, Y_test, Y_test]

for i in range(4):
    sample = samples[i]
    sample['label'] = targets[i]
    sample.to_csv("../data/processed/" + filenames[i])


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pickle
import string
import re
import nltk
from nltk.stem import WordNetLemmatizer
from sklearn import preprocessing
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline

# Metrics
from sklearn import metrics
from sklearn.model_selection import cross_val_score
from sklearn.metrics import ConfusionMatrixDisplay, classification_report, roc_curve, fbeta_score
import scikitplot as skplt

# Models
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import tree
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier

from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import make_pipeline


# Ploting functions for the models
def confusion_matrix(titles_options = None, test = None, pred = None, labels = None):
    '''
    Prints confusion matrix making the predictions
    :param titles_options:
    :param test:
    :param pred:
    :param labels:
    :return:
    '''
    for title, normalize in titles_options:
        disp = metrics.ConfusionMatrixDisplay.from_predictions(
            test,
            pred,
            display_labels=labels,
            cmap=plt.cm.Blues,
            normalize=normalize
        )
        disp.ax_.set_title(title)

        print(title)
        print(disp.confusion_matrix)

    plt.show()

def model_curves_summary(test = None, pred = None, label = 'Model'):
    '''
    Prints a plot report with four different plots: ROC curve, PR curve,
    Gain curve, Lift curve
    :param test: Real values
    :param pred: Predicted values
    :param label: Model name
    '''
    fig, axes = plt.subplots(2, 2, figsize=(15, 10), sharey=False)
    fig.suptitle('Curvas del modelo ' + label)

    # ROC CURVE
    fpr, tpr, thresholds = metrics.roc_curve(test, pred[:,1])
    # plot the roc curve for the model
    axes[0,0].plot([0,1], [0,1], linestyle='--', label='No Skill')
    axes[0,0].plot(fpr, tpr, marker='.', label=label)
    # axis labels
    axes[0,0].set_title('Curva ROC')
    axes[0,0].set(xlabel='False Positive Rate', ylabel='True Positive Rate')
    axes[0,0].legend()
    axes[0,0].grid()

    # Precission-Recall CURVE
    # calculate pr-curve
    precision, recall, thresholds = metrics.precision_recall_curve(test, pred[:,1])
    no_skill = len(test[test==1]) / len(test)
    axes[0,1].plot([0,1], [no_skill,no_skill], linestyle='--', label='No Skill')
    axes[0,1].plot(recall, precision, marker='.', label=label)
    # axis labels
    axes[0,1].set_title('Curva Precission-Recall')
    axes[0,1].set(xlabel='Precission', ylabel='Recall')
    axes[0,1].legend()
    axes[0,1].grid()

    # GAIN CURVE
    skplt.metrics.plot_cumulative_gain(test, pred, ax = axes[1,0])

    # LIFT CURVE
    skplt.metrics.plot_lift_curve(test, pred, ax = axes[1,1])


# Metrics list
models_report_df = pd.DataFrame(columns=['Model', 'Accuracy', 'Precission', 'Recall', 'F-score', 'AUC'])

# Model final list for later use
models_collection = {}

def model_train_test(model, traind, traind_labels, testd, testd_labels, data_info, i):
    model_name = str(type(model).__name__) + data_info

    # Training model
    print("Training model... " + model_name)
    model.fit(traind, traind_labels)

    # Prediction with model
    print("Predicting with model... " + model_name)
    y_pred = model.predict(testd)
    
    # Model information
    models_report_df.loc[i] = [
        model_name,
        metrics.accuracy_score(testd_labels, y_pred),
        metrics.precision_score(testd_labels, y_pred),
        metrics.recall_score(testd_labels, y_pred),
        metrics.f1_score(testd_labels, y_pred),
        metrics.roc_auc_score(testd_labels, y_pred)
    ]

    # Saving the model
    models_collection[model_name] = model
    
    print("Ended evaluation of model... " + model_name)


# Data preprocessed with BOW only
train_bow = pd.read_csv("../data/processed/train_bow.csv")
test_bow = pd.read_csv("../data/processed/test_bow.csv")

Y_train_bow = train_bow['label']
X_train_bow = train_bow.drop(['label'], axis=1)

Y_test_bow = test_bow['label']
X_test_bow = test_bow.drop(['label'], axis=1)

# Data preprocessed with BOW and TFD 
train_tfd = pd.read_csv("../data/processed/train_tfd.csv")
test_tfd = pd.read_csv("../data/processed/test_tfd.csv")

Y_train_tfd = train_tfd['label']
X_train_tfd = train_tfd.drop(['label'], axis=1)

Y_test_tfd = test_tfd['label']
X_test_tfd = test_tfd.drop(['label'], axis=1)


models = [tree.DecisionTreeClassifier(), RandomForestClassifier(), MultinomialNB(), GradientBoostingClassifier()]

for i, m in enumerate(models):
    model_train_test(m, X_train_bow, Y_train_bow, X_test_bow, Y_test_bow, "_bow", i)

Training model... DecisionTreeClassifier_bow
Predicting with model... DecisionTreeClassifier_bow
Ended evaluation of model... DecisionTreeClassifier_bow
Training model... RandomForestClassifier_bow
Predicting with model... RandomForestClassifier_bow
Ended evaluation of model... RandomForestClassifier_bow
Training model... MultinomialNB_bow
Predicting with model... MultinomialNB_bow
Ended evaluation of model... MultinomialNB_bow
Training model... GradientBoostingClassifier_bow
Predicting with model... GradientBoostingClassifier_bow
Ended evaluation of model... GradientBoostingClassifier_bow


models = [tree.DecisionTreeClassifier(), RandomForestClassifier(), MultinomialNB(), GradientBoostingClassifier(), LogisticRegression()]

for i, m in enumerate(models):
    model_train_test(m, X_train_tfd, Y_train_tfd, X_test_tfd, Y_test_tfd, "_tfid", i+4)

Training model... DecisionTreeClassifier_tfid
Predicting with model... DecisionTreeClassifier_tfid
Ended evaluation of model... DecisionTreeClassifier_tfid
Training model... RandomForestClassifier_tfid
Predicting with model... RandomForestClassifier_tfid
Ended evaluation of model... RandomForestClassifier_tfid
Training model... MultinomialNB_tfid
Predicting with model... MultinomialNB_tfid
Ended evaluation of model... MultinomialNB_tfid
Training model... GradientBoostingClassifier_tfid
Predicting with model... GradientBoostingClassifier_tfid
Ended evaluation of model... GradientBoostingClassifier_tfid
Training model... LogisticRegression_tfid
Predicting with model... LogisticRegression_tfid
Ended evaluation of model... LogisticRegression_tfid


models_report_df


titles_options = [
    ("Matriz de confusión", None),
    ("Matriz de confusión normalizada", "true"),
]
confusion_matrix(titles_options, Y_test_tfd, models_collection['RandomForestClassifier_tfid'].predict(X_test_tfd), 
                            ['FAKE', 'REAL'])

Matriz de confusión
[[402  73]
 [103 373]]
Matriz de confusión normalizada
[[0.84631579 0.15368421]
 [0.21638655 0.78361345]]


titles_options = [
    ("Matriz de confusión", None),
    ("Matriz de confusión normalizada", "true"),
]
confusion_matrix(titles_options, Y_test_bow, models_collection['MultinomialNB_bow'].predict(X_test_bow), 
                            ['FAKE', 'REAL'])

Matriz de confusión
[[387  88]
 [ 84 392]]
Matriz de confusión normalizada
[[0.81473684 0.18526316]
 [0.17647059 0.82352941]]


model_curves_summary(Y_test_tfd, models_collection['RandomForestClassifier_tfid'].predict_proba(X_test_tfd), "RandomForestClassifier")


model_curves_summary(Y_test_bow, models_collection['MultinomialNB_bow'].predict_proba(X_test_bow), "MultinomialNB")


def cleaning_data(tlt_df):
    # Translator to clean data 
    translator = str.maketrans('', '', string.punctuation)

    # Word lemmatizer
    wordnet_lemmatizer = WordNetLemmatizer()

    # Stopwords
    stopwords = nltk.corpus.stopwords.words('english')

    return tlt_df.map(lambda x : " ".join(
        [wordnet_lemmatizer.lemmatize(word, pos="v") for word in 
        re.sub(r"[^a-zA-Z0-9]", " ", x.strip().lower().translate(translator)).split() if not word in stopwords])
)

estimator_model =  make_pipeline(CountVectorizer(), MultinomialNB())


train_full = pd.read_csv("../data/interim/train.csv")
Y_train = train_full['label'].str.replace("FAKE", "0").str.replace("REAL", "1").map(lambda x : int(x))
X_train = cleaning_data(train_full.drop(['label'], axis=1)['title'])


param_grid = {
    'countvectorizer__binary': [True, False],
    "multinomialnb__alpha": [0.01, 0.05, 0.1, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 1, 1.25, 1.5, 1.75, 2],
    "multinomialnb__fit_prior": [True, False]
}

search = GridSearchCV(estimator_model, param_grid, scoring="recall")


result = search.fit(X_train, Y_train)


print('Best Score: %s' % result.best_score_)
print('Best Hyperparameters: %s' % result.best_params_)

Best Score: 0.847495361781076
Best Hyperparameters: {'countvectorizer__binary': True, 'multinomialnb__alpha': 1.75, 'multinomialnb__fit_prior': True}


final_model = Pipeline(
    [('vector', CountVectorizer(binary=True)),
    ('model', MultinomialNB(alpha=0.75, fit_prior=False))]
)

final_model.fit(X_train, Y_train)

Pipeline(steps=[('vector', CountVectorizer(binary=True)),
                ('model', MultinomialNB(alpha=0.75, fit_prior=False))])


test_full = pd.read_csv("../data/interim/test.csv")
Y_test = test_full['label'].str.replace("FAKE", "0").str.replace("REAL", "1").map(lambda x : int(x))
X_test = cleaning_data(test_full.drop(['label'], axis=1)['title'])


Y_pred = final_model.predict(X_test)


metrics.accuracy_score(Y_test, Y_pred)

0.8128286014721346


titles_options = [
    ("Matriz de confusión", None),
    ("Matriz de confusión normalizada", "true"),
]
confusion_matrix(titles_options, Y_test, Y_pred, 
                            ['FAKE', 'REAL'])

Matriz de confusión
[[369 106]
 [ 72 404]]
Matriz de confusión normalizada
[[0.77684211 0.22315789]
 [0.1512605  0.8487395 ]]


model_curves_summary(Y_test, final_model.predict_proba(X_test), "Final Model")


file_name = "../models/fake_class_model.sav"
pickle.dump(final_model, open(file_name, "wb"))

C:\Users\carviagu\anaconda3\envs\NSD\lib\site-packages\pyLDAvis\_prepare.py:246: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only.
  default_term_info = default_term_info.sort_values(


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from gensim.parsing.preprocessing import preprocess_string
from gensim.corpora import Dictionary
from gensim.models import ldamodel

import pyLDAvis
import pyLDAvis.gensim_models


test_full = pd.read_csv("../data/interim/test.csv")[['title', 'label']]
test_full['label'] = test_full['label'].str.replace("FAKE", "0").str.replace("REAL", "1").map(lambda x : int(x))
test_full['title'] = test_full.title.apply(preprocess_string)


dictionary = Dictionary(test_full['title'])
bow_corpus = [dictionary.doc2bow(text) for text in test_full['title']]


ldamodel = ldamodel.LdaModel(bow_corpus, num_topics=15, id2word = dictionary, passes=20)


pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim_models.prepare(ldamodel, bow_corpus, dictionary)
vis

C:\Users\carviagu\anaconda3\envs\NSD\lib\site-packages\pyLDAvis\_prepare.py:246: FutureWarning: In a future version of pandas all arguments of DataFrame.drop except for the argument 'labels' will be keyword-only.
  default_term_info = default_term_info.sort_values(

	Unnamed: 0	title	text	label
0	8476	You Can Smell Hillary’s Fear	Daniel Greenfield, a Shillman Journalism Fello...	FAKE
1	10294	Watch The Exact Moment Paul Ryan Committed Pol...	Google Pinterest Digg Linkedin Reddit Stumbleu...	FAKE
2	3608	Kerry to go to Paris in gesture of sympathy	U.S. Secretary of State John F. Kerry said Mon...	REAL
3	10142	Bernie supporters on Twitter erupt in anger ag...	— Kaydee King (@KaydeeKing) November 9, 2016 T...	FAKE
4	875	The Battle of New York: Why This Primary Matters	It's primary day in New York and front-runners...	REAL

	Unnamed: 0	title	text	label
1342	8196	Voter Dreading Being Sent Over To Visibly Stup...	Trump Raises Concern Over Members Of Urban Com...	0
1251	30	Abortion bill dropped amid concerns of female ...	This item has been updated.\n\nHouse Republica...	1
804	7216	Internet Is On Fire With Speculation That Pode...	We Are Change \nEmails revealed by Wikileaks f...	0
47	587	Senate race rankings: Dems attack as GOP lays ...	The move would make it easier for the Trump ad...	1
3888	103	Starbucks baristas stop writing 'Race Together...	In a marketing fiasco that could rank right up...	1
...	...	...	...	...
502	1118	Romney's timely Trump trolling	**Want FOX News First in your inbox every day?...	1
2824	4423	Megyn Kelly interrogates Tom Cotton on Iran le...	On Monday, Cotton drafted a letter — signed by...	1
1768	9347	CONTROVERSIAL NEW ‘ANTI-FAMINE’ GMO POTATO STR...	Home › HEALTH \| US NEWS › CONTROVERSIAL NEW ‘A...	0
4116	6239	Meter Reader Knocks On Man’s Door, Reveals Sin...	Share This \nWhen a man heard a knock on his d...	0
4944	9843	‘On Contact’: Chris Hedges and Medea Benjamin ...	‘On Contact’: Chris Hedges and Medea Benjamin ...	0

	Model	Accuracy	Precission	Recall	F-score	AUC
0	DecisionTreeClassifier_bow	0.763407	0.757700	0.775210	0.766355	0.763395
1	RandomForestClassifier_bow	0.811777	0.835214	0.777311	0.805223	0.811813
2	MultinomialNB_bow	0.819138	0.816667	0.823529	0.820084	0.819133
3	GradientBoostingClassifier_bow	0.762355	0.881098	0.607143	0.718905	0.762519
4	DecisionTreeClassifier_tfid	0.740273	0.717268	0.794118	0.753739	0.740217
5	RandomForestClassifier_tfid	0.814932	0.836323	0.783613	0.809111	0.814965
6	MultinomialNB_tfid	0.807571	0.781190	0.855042	0.816449	0.807521
7	GradientBoostingClassifier_tfid	0.761304	0.842975	0.642857	0.729440	0.761429
8	LogisticRegression_tfid	0.809674	0.840647	0.764706	0.800880	0.809721

Fake News Classifier Modelling

Carlos Viñals Guitart (carlos.vinals@cunef.edu)

Ciencia de Datos para la Gestión de Información No Estructurada - MUCD

1.0 Data Analysis and Exploration

Lectura y análisis preliminar¶

División de train y test¶

Análisis detallado de los datos¶

Procesado¶

2.0 Data Preprocessing

Lectura de datos¶

Procesado de los datos¶

Creación de variables del modelo¶

Guardado de los datos¶

3.0 Models

4.0 Topic Modelling

	index	percentaje	label
0	REAL	50.055249	3171
1	FAKE	49.944751	3164

	index	percentaje	label
0	REAL	50.055721	2695
1	FAKE	49.944279	2689